%matplotlib inline
import pandas as pd
import numpy as np
import datetime as dt
import portfolioopt as port
import blacklitterman as bl
from plotly import __version__
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
from plotly import tools
import plotly.graph_objs as go
init_notebook_mode(connected=True)
Leonardo Barros 333799
Pedro Mattos 333791
Veridiana Fonseca 333801
def plotLines(series, legends, title):
data = []
i=0
for serie in series:
trace = go.Scatter(
x = pd.to_datetime(serie.index.values),
y = serie / 1000,
name = legends[i]
)
data.append(trace)
i += 1
layout = dict(title = title,
xaxis = dict(title = 'Date'),
yaxis = dict(title = 'Return'),
)
fig = dict(data=data, layout=layout)
iplot(fig)
# assets = [
# 'PETR4',
# 'VALE5',
# 'IBOV',
# ]
# dfs = {}
# for name in assets:
# dfs[name] = pd.read_csv(name + '.txt', sep=',')
# dfs[name]['Date'] = dfs[name]['Date'].apply(pd.to_datetime)
# dfs[name].set_index('Date',inplace=True)
# dfs[name].sort_index(inplace=True)
# dfs[name][name] = dfs[name]['Close'].pct_change()
assets = [
'IMAB',
# 'IMAS',
'USDBRL',
'IBOV',
]
dfs = {}
for name in assets:
dfs[name] = pd.read_csv(name + '.csv', sep=',')
dfs[name]['Date'] = dfs[name]['Date'].apply(pd.to_datetime)
dfs[name].set_index('Date',inplace=True)
dfs[name].sort_index(inplace=True)
dfs[name]['Close'] = dfs[name][name]
dfs[name][name] = dfs[name][name].fillna(method='ffill').pct_change()
returns = []
for name in assets:
returns.append(dfs[name][name])
returns = pd.concat(returns,axis=1)
prices = []
for name in assets:
dfs[name][name] = dfs[name]['Close']
prices.append(dfs[name][name])
prices = pd.concat(prices,axis=1)
returns = prices.fillna(method='ffill').pct_change()
marketCap = prices
returns.head()
cdi = pd.read_csv('CDI.csv',dayfirst=True)
cdi['Data'] = pd.to_datetime(cdi['Data'], dayfirst=True)
cdi.set_index('Data',inplace=True)
# cdi
cdi.sort_index(inplace=True)
cdi = cdi.reindex(index=returns.index,method='ffill')
# cdi.head()
# cdi['CDI']
for asset in assets:
cdi[asset] = cdi['CDI']
cdi.drop('CDI', axis=1, inplace=True)
# returns = returns.subtract(cdi)
returns.head()
series = []
for asset in assets:
series.append(((returns[asset] + 1).cumprod()-1)*1000)
plotLines(series, assets, 'Assets')
Para Rebalances
monthlyDates = pd.DataFrame(data=returns.index, index = returns.index)
monthlyDates = monthlyDates.resample('M').min().set_index('Date').index
monthlyDates
quarterlyDates = pd.DataFrame(data=returns.index, index = returns.index)
quarterlyDates = quarterlyDates.resample('Q').min().set_index('Date').index
quarterlyDates
annualDates = pd.DataFrame(data=returns.index, index = returns.index)
annualDates = annualDates.resample('A').min().set_index('Date').index
annualDates
def generateEqualWeights(baseDf, rebalDates, prices, assets, returns):
weights = pd.DataFrame(np.nan,index=rebalDates, columns = assets)
weights.loc[rebalDates] = (1/3)
# return weights.fillna(method='ffill')
return weights
equalWeights = {}
equalWeights['monthly'] = generateEqualWeights(returns,monthlyDates, prices, assets, returns)
equalWeights['quarterly'] = generateEqualWeights(returns,quarterlyDates, prices, assets, returns)
equalWeights['annual'] = generateEqualWeights(returns,annualDates, prices, assets, returns)
equalWeights['annual']
For each rebalance date we need to estimate a covariance matrix using past data and generate a efficient frontier to find the minimum variance portfolio.
def generateMinimumVarianceWeights(baseDf, rebalDates, prices, assets, returns):
weights = pd.DataFrame(np.nan,index=rebalDates, columns = assets)
for rebalDate in rebalDates:
temp = port.min_var_portfolio(returns.loc[:rebalDate].cov().fillna(0), allow_short=False)
# index = returns.index.get_loc(rebalDate)
# startIndex = int(np.array([0,index-30]).min())
# temp = port.min_var_portfolio(returns.iloc[startIndex:index].cov().fillna(0), allow_short=False)
weights.loc[rebalDate] = temp
return weights
minimumVariance = {}
minimumVariance['monthly'] = generateMinimumVarianceWeights(returns,monthlyDates, prices, assets, returns)
minimumVariance['quarterly'] = generateMinimumVarianceWeights(returns,quarterlyDates, prices, assets, returns)
minimumVariance['annual'] = generateMinimumVarianceWeights(returns,annualDates, prices, assets, returns)
minimumVariance['annual']
def generateTangencyWeights(baseDf, rebalDates, prices, assets, returns):
weights = pd.DataFrame(np.nan,index=rebalDates, columns = assets)
for rebalDate in rebalDates:
# print(returns.loc[:rebalDate].mean().fillna(0.1))
# print(returns.loc[:rebalDate].cov().fillna(0))
temp = port.tangency_portfolio(exp_rets=returns.loc[:rebalDate].mean().fillna(0.1),cov_mat=returns.loc[:rebalDate].cov().fillna(0), allow_short=False)
weights.loc[rebalDate] = temp
return weights
tangency = {}
tangency['monthly'] = generateTangencyWeights(returns,monthlyDates, prices, assets, returns)
tangency['quarterly'] = generateTangencyWeights(returns,quarterlyDates, prices, assets, returns)
tangency['annual'] = generateTangencyWeights(returns,annualDates, prices, assets, returns)
tangency['annual']
# print(equalWeights['monthly'].loc[returns.index.values[0]])
def backtest(weights,returns):
qtd = pd.DataFrame(index=returns.index, columns=returns.columns)
for date in returns.index.values:
prevIndex = qtd.index.get_loc(date)-1
if date in weights.index:
#se houver pesos
# o financeiro de fechamento de hoje sera o total do financeiro de ontem vezes
# o novo peso, vezes o retorno de hoje
if prevIndex> 0 :
prevCloseFin = qtd.iloc[prevIndex].sum()
qtd.loc[date] = prevCloseFin * weights.loc[date] * (returns.loc[date] + 1)
else:
prevCloseFin = 1000
qtd.loc[date] = prevCloseFin * weights.loc[date] #* (returns.loc[date] + 1)
else:
# caso nao haja rebalance o financeiro de hoje sera o de ontem vezes o retorno de hoje
qtd.loc[date] = qtd.iloc[prevIndex] * (returns.loc[date] + 1)
return qtd
# print(returns.loc[date])
rebalanceFrequencies = ['monthly', 'quarterly', 'annual']
series = []
for freq in rebalanceFrequencies:
series.append(backtest(equalWeights[freq],returns).sum(axis=1))
plotLines(series, rebalanceFrequencies, 'Equal Weight Strategy')
series[0].index.values
rebalanceFrequencies = ['monthly', 'quarterly', 'annual']
series = []
for freq in rebalanceFrequencies:
series.append(backtest(minimumVariance[freq],returns).sum(axis=1))
plotLines(series, rebalanceFrequencies, 'Minimum Variance Strategy')
rebalanceFrequencies = ['monthly', 'quarterly', 'annual']
series = []
for freq in rebalanceFrequencies:
series.append(backtest(tangency[freq],returns).sum(axis=1))
plotLines(series, rebalanceFrequencies, 'Tangency Strategy')
strategies = [equalWeights, minimumVariance, tangency]
strategiesNames = ['Equal Weights', 'Minimum Variance', 'Tangency']
freq = 'monthly'
series = []
for strat in strategies:
series.append(backtest(strat[freq],returns).sum(axis=1))
plotLines(series, strategiesNames, 'Monthly Rebalance Strategies')
strategies = [equalWeights, minimumVariance, tangency]
freq = 'quarterly'
series = []
for strat in strategies:
series.append(backtest(strat[freq],returns).sum(axis=1))
plotLines(series, strategiesNames, 'Quarterly Rebalance Strategies')
strategies = [equalWeights, minimumVariance, tangency]
freq = 'annual'
series = []
for strat in strategies:
series.append(backtest(strat[freq],returns).sum(axis=1))
plotLines(series, strategiesNames, 'Annual Rebalance Strategies')
# resample each series
def resampleEach(df):
n = df.shape[0]
newDf = df.copy()
for col in df.columns:
newDf[col] = df[col].sample(n, replace=True).values
return newDf.fillna(method='ffill')
listOfDfs = []
average = []
for n in range(0,1000):
temp = resampleEach(returns)
listOfDfs.append(temp)
average.append(temp.mean(axis=1).mean())
var5 = np.percentile(np.array(average),0.05, interpolation='higher')
varReturn5 = listOfDfs[average.index(var5)]
var1 = np.percentile(np.array(average),0.01, interpolation='lower')
varReturn1 = listOfDfs[average.index(var1)]
series = []
for asset in assets:
series.append(((returns[asset] + 1).cumprod()-1)*1000)
plotLines(series, assets, 'Assets - Real Scenario')
series = []
for asset in assets:
series.append(((varReturn5[asset] + 1).cumprod()-1)*1000)
plotLines(series, assets, 'Assets - 5% VaR')
series = []
for asset in assets:
series.append(((varReturn1[asset] + 1).cumprod()-1)*1000)
plotLines(series, assets, 'Assets - 1% VaR')
Target: Find which frequency works best
Recalculate weights
var5tangency = {}
var5tangency['monthly'] = generateTangencyWeights(varReturn5,monthlyDates, prices, assets, varReturn5)
var5tangency['quarterly'] = generateTangencyWeights(varReturn5,quarterlyDates, prices, assets, varReturn5)
var5tangency['annual'] = generateTangencyWeights(varReturn5,annualDates, prices, assets, varReturn5)
series = []
for freq in rebalanceFrequencies:
series.append(backtest(var5tangency[freq],varReturn5).sum(axis=1))
plotLines(series, rebalanceFrequencies, 'Tangency Strategy - 5% VaR')
Recalculate weights
var1tangency = {}
var1tangency['monthly'] = generateTangencyWeights(varReturn1,monthlyDates, prices, assets, varReturn1)
var1tangency['quarterly'] = generateTangencyWeights(varReturn1,quarterlyDates, prices, assets, varReturn1)
var1tangency['annual'] = generateTangencyWeights(varReturn1,annualDates, prices, assets, varReturn1)
series = []
for freq in rebalanceFrequencies:
series.append(backtest(var1tangency[freq],varReturn1).sum(axis=1))
plotLines(series, rebalanceFrequencies, 'Tangency Strategy - 1% VaR')
var5quarterly = {}
var5quarterly['EqualWeights'] = generateEqualWeights(varReturn5,monthlyDates, prices, assets, varReturn1)
var5quarterly['MinimumVariance'] = generateMinimumVarianceWeights(varReturn1,quarterlyDates, prices, assets, varReturn1)
var5quarterly['Tangency'] = generateTangencyWeights(varReturn1,annualDates, prices, assets, varReturn1)
strategies = ['EqualWeights', 'MinimumVariance','Tangency']
freq = 'quarterly'
series = []
for strat in strategies:
series.append(backtest(var5quarterly[strat],varReturn1).sum(axis=1))
plotLines(series, strategiesNames, 'Quarterly Rebalance Strategies - VaR 5% Scenario')
var1quarterly = {}
var1quarterly['EqualWeights'] = generateEqualWeights(varReturn1,monthlyDates, prices, assets, varReturn1)
var1quarterly['MinimumVariance'] = generateMinimumVarianceWeights(varReturn1,quarterlyDates, prices, assets, varReturn1)
var1quarterly['Tangency'] = generateTangencyWeights(varReturn1,annualDates, prices, assets, varReturn1)
strategies = ['EqualWeights', 'MinimumVariance','Tangency']
freq = 'quarterly'
series = []
for strat in strategies:
series.append(backtest(var1quarterly[strat],varReturn1).sum(axis=1))
plotLines(series, strategiesNames, 'Quarterly Rebalance Strategies - VaR 1% Scenario')
fig = tools.make_subplots(rows=3, cols=1, subplot_titles=list(var1quarterly.keys()))
i=1
for strat in var1quarterly:
for col in var1quarterly[strat].columns:
trace = go.Scatter(
x = pd.to_datetime(var1quarterly[strat][col].index.values),
y = var1quarterly[strat][col],
name = col,
)
fig.append_trace(trace, i, 1)
i += 1
# layout = dict(title = title,
# xaxis = dict(title = 'Date'),
# yaxis = dict(title = 'Return'),
# )
# fig['layout'].update(height=600, width=600, title='i <3 subplots')
fig['layout'].update(title='Weights - VaR 1% Scenario')
iplot(fig)
series = []
for asset in assets:
series.append(((varReturn1[asset] + 1).cumprod()-1)*1000)
plotLines(series, assets, 'Assets - 1% VaR')
help(port.tangency_portfolio)
P.shape
delta = 2.5
tau = 0.05
weq = np.array(marketCap.iloc[-1])
sigma = np.array(returns.cov())
P = np.array([0.1,0,0])
Q = np.array([0.05])
Omega = np.array([
[0.1,0,0],
[0,0.05,0],
[0,0,0.02],
])
Omega = np.array([
0.01
])
# Omega = sigma
x = bl.blacklitterman(delta, weq, sigma, tau, P, Q, Omega)
help(bl.blacklitterman)
(returns+1).cumprod().plot()
returns.mean()
returns.cov()
x = pd.DataFrame(np.transpose(np.array([[1,2,3,4,5],[6,7,8,9,10]])))
x
x.sample(n=5)